package com.lzx.crawler;

import ch.qos.logback.core.util.FileUtil;
import com.lzx.crawler.entity.IPEntity;
import com.lzx.crawler.util.CrawlerUtil;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;

import java.util.LinkedList;
import java.util.List;

@SpringBootApplication
public class CrawlerApplication {

    public static void main(String[] args) {
        SpringApplication.run(CrawlerApplication.class, args);
        try{
//            for (int i = 1;i < 10;i++){
            String url = "https://www.kuaidaili.com/free/inha/";
            List<IPEntity> ips = getIP(url);
            System.out.println("获取的ip有"+ips.size()+"个，分别是:" + ips);

            testIP(ips);
            System.out.println("有效的ip有"+ips.size()+"个，分别是:" + ips);
            FileUtil.
//                Thread.sleep(1000*60);
//            }
        }catch (Exception e){
            e.printStackTrace();
        }
    }
    public static List<IPEntity>  getIP(String url){
        List<IPEntity> ips = new LinkedList<IPEntity>();
        CloseableHttpResponse response = null;
        CloseableHttpClient httpClient = null;
        try{

            HttpGet get = new HttpGet(url);
            get.setHeader("user-agent"," Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36");
            httpClient = HttpClientBuilder.create().build();
            response = httpClient.execute(get);

            if(response.getStatusLine().getStatusCode() == 200) {
                HttpEntity entity = response.getEntity();
                String html = EntityUtils.toString(entity,"gb2312");
                Document doc = Jsoup.parse(html);
                Elements ulList = doc.select("table");
                Elements liList = ulList.get(0).select("tr");
                for(int i = 1;i < liList.size();i++) {
                    String [] strings = liList.get(i).text().split(" ");
                    IPEntity ip = new IPEntity(strings[0],new Integer(strings[1]),strings[3]);
                    ips.add(ip);
                }
                EntityUtils.consume(response.getEntity());
            }else{
                EntityUtils.consume(response.getEntity());
            }

        }catch (Exception e){
            e.printStackTrace();
        }finally {
            try{
                if(response != null){
                    response.close();
                }
                if(httpClient != null){
                    httpClient.close();
                }
            }catch (Exception e){
                e.printStackTrace();
            }
        }
        return ips;
    }
    public static List<IPEntity> testIP(List<IPEntity> ips){
        for(int i = 0;i < ips.size();i++){
            if(!CrawlerUtil.runTest(ips.get(i))){
                System.out.println("无效!  "+ips.get(i));
                ips.remove(i);
            }else {
                System.out.println("有效!  "+ips.get(i));
            }
        }
        return ips;
    }

}
